{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "fer2013.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "cVbLZURCveTi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%tensorflow_version 1.x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "2nz38mJZXN_P",
        "outputId": "92320703-51dd-42d6-fa99-2d5b6c3d2c21",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.layers import *\n",
        "from tensorflow.python.lib.io import file_io\n",
        "\n",
        "%matplotlib inline\n",
        "\n",
        "import keras\n",
        "from keras import backend as K\n",
        "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
        "from keras.models import load_model\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from keras.utils import plot_model\n",
        "from sklearn.metrics import *\n",
        "from keras.engine import Model\n",
        "from keras.layers import Input, Flatten, Dense, Activation, Conv2D, MaxPool2D, BatchNormalization, Dropout, MaxPooling2D\n",
        "import skimage\n",
        "from skimage.transform import rescale, resize"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "EjKPXZ3TX3Jb",
        "outputId": "9b02393a-903b-4ce0-b9b3-cb5df9bfb6fa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l2DlU8lZeOg7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Resize_pixelsize = 197"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xSJv4GKReUwb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Function that reads the data from the csv file, increases the size of the images and returns the images and their labels\n",
        "    # dataset: Data path\n",
        "def get_data(dataset):\n",
        "    \n",
        "    file_stream = file_io.FileIO(dataset, mode='r')\n",
        "    data = pd.read_csv(file_stream)\n",
        "    data[' pixels'] = data[' pixels'].apply(lambda x: [int(pixel) for pixel in x.split()])\n",
        "    X, Y = data[' pixels'].tolist(), data['emotion'].values\n",
        "    X = np.array(X, dtype='float32').reshape(-1,48,48,1)\n",
        "    X = X/255.0\n",
        "   \n",
        "    X_res = np.zeros((X.shape[0], Resize_pixelsize,Resize_pixelsize,3))\n",
        "    for ind in range(X.shape[0]): \n",
        "        sample = X[ind]\n",
        "        sample = sample.reshape(48, 48)\n",
        "        image_resized = resize(sample, (Resize_pixelsize, Resize_pixelsize), anti_aliasing=True)\n",
        "        X_res[ind,:,:,:] = image_resized.reshape(Resize_pixelsize,Resize_pixelsize,1)\n",
        "\n",
        "    Y_res = np.zeros((Y.size, 7))\n",
        "    Y_res[np.arange(Y.size),Y] = 1    \n",
        "    \n",
        "    return  X, X_res, Y_res"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j8FT39NZgDHP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dev_dataset_dir = '/content/drive/My Drive/cs230 project/collab/fer2013/dev.csv'\n",
        "test_dataset_dir = '/content/drive/My Drive/cs230 project/collab/fer2013/test.csv'\n",
        "\n",
        "X_dev, X_res_dev, Y_dev   = get_data(dev_dataset_dir)\n",
        "X_test, X_res_test, Y_test   = get_data(test_dataset_dir)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XmmvoAzOgQl4",
        "colab_type": "code",
        "outputId": "8cdc24da-8801-42d4-bc82-950ca2df38d7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 683
        }
      },
      "source": [
        "model = load_model('/content/drive/My Drive/cs230 project/models/soa-SGD_LR_0.01000-EPOCHS_300-BS_128-DROPOUT_0.3test_acc_0.663.h5')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:541: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:190: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:197: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:203: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:207: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:216: The name tf.is_variable_initialized is deprecated. Please use tf.compat.v1.is_variable_initialized instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:223: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:2041: The name tf.nn.fused_batch_norm is deprecated. Please use tf.compat.v1.nn.fused_batch_norm instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:148: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4267: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3733: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:793: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3576: The name tf.log is deprecated. Please use tf.math.log instead.\n",
            "\n",
            "WARNING:tensorflow:From /tensorflow-1.15.0/python3.6/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1033: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1020: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ABbKc8-Eq64L",
        "colab_type": "code",
        "outputId": "d872d7a4-3b29-43ac-de66-fa0dc523775c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "print('\\n# Evaluate on dev data')\n",
        "results_dev = model.evaluate(X_dev,Y_dev)\n",
        "print('dev loss, dev acc:', results_dev)\n",
        "\n",
        "print('\\n# Evaluate on test data')\n",
        "results_test = model.evaluate(X_test,Y_test)\n",
        "print('test loss, test acc:', results_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "# Evaluate on dev data\n",
            "3589/3589 [==============================] - 7s 2ms/step\n",
            "dev loss, dev acc: [1.0376124443799282, 0.6358317080049602]\n",
            "\n",
            "# Evaluate on test data\n",
            "3589/3589 [==============================] - 0s 82us/step\n",
            "test loss, test acc: [0.9683632022260324, 0.6631373641765957]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ED86bdWtpdCL",
        "colab_type": "code",
        "outputId": "7e47dc92-d1c8-498d-e0d4-e14e0ac0bd6c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "model2 = load_model('/content/drive/My Drive/cs230 project/models/soa-SGD_LR_0.01000-EPOCHS_300-BS_128-DROPOUT_0.4test_acc_0.657.h5')\n",
        "print('\\n# Evaluate on dev data')\n",
        "results_dev = model2.evaluate(X_dev,Y_dev)\n",
        "print('dev loss, dev acc:', results_dev)\n",
        "\n",
        "print('\\n# Evaluate on test data')\n",
        "results_test = model2.evaluate(X_test,Y_test)\n",
        "print('test loss, test acc:', results_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "# Evaluate on dev data\n",
            "3589/3589 [==============================] - 0s 120us/step\n",
            "dev loss, dev acc: [1.0297004918794375, 0.6514349401030377]\n",
            "\n",
            "# Evaluate on test data\n",
            "3589/3589 [==============================] - 0s 88us/step\n",
            "test loss, test acc: [0.9837366009230201, 0.6570075230035121]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wtkh5xsNvbBG",
        "colab_type": "code",
        "outputId": "89355d78-6073-468a-d013-d10e1b1f2dc6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        }
      },
      "source": [
        "Resnet_model = load_model('/content/drive/My Drive/cs230 project/models/tl/ResNet-BEST-73.2.h5')\n",
        "print('\\n# Evaluate on dev data')\n",
        "results_dev = Resnet_model.evaluate(X_res_dev,Y_dev)\n",
        "print('dev loss, dev acc:', results_dev)\n",
        "\n",
        "print('\\n# Evaluate on test data')\n",
        "results_test = Resnet_model.evaluate(X_res_test,Y_test)\n",
        "print('test loss, test acc:', results_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4271: The name tf.nn.avg_pool is deprecated. Please use tf.nn.avg_pool2d instead.\n",
            "\n",
            "\n",
            "# Evaluate on dev data\n",
            "3589/3589 [==============================] - 9s 3ms/step\n",
            "dev loss, dev acc: [1.0430927014211386, 0.6960156032404019]\n",
            "\n",
            "# Evaluate on test data\n",
            "3589/3589 [==============================] - 8s 2ms/step\n",
            "test loss, test acc: [0.9163703456932072, 0.732237392047814]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7fNpbezx0lek",
        "colab_type": "code",
        "outputId": "95d7002b-aa58-4e76-d54f-f00b1558e294",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "Resnet_model_wcw = load_model('/content/drive/My Drive/cs230 project/models/tl/ResNet-BEST-WCW-0.677.h5')\n",
        "print('\\n# Evaluate on dev data')\n",
        "results_dev = Resnet_model_wcw.evaluate(X_res_dev,Y_dev)\n",
        "print('dev loss, dev acc:', results_dev)\n",
        "\n",
        "print('\\n# Evaluate on test data')\n",
        "results_test = Resnet_model_wcw.evaluate(X_res_test,Y_test)\n",
        "print('test loss, test acc:', results_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "# Evaluate on dev data\n",
            "3589/3589 [==============================] - 10s 3ms/step\n",
            "dev loss, dev acc: [1.2324885091980242, 0.6698244636388966]\n",
            "\n",
            "# Evaluate on test data\n",
            "3589/3589 [==============================] - 8s 2ms/step\n",
            "test loss, test acc: [1.1428996915603356, 0.6767901922707174]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "opo1eF3swqme",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Senet_model = load_model('/content/drive/My Drive/cs230 project/models/tl/SeNet50-BEST-69.8.h5')\n",
        "# print('\\n# Evaluate on dev data')\n",
        "# results_dev = Senet_model.evaluate(X_res_dev,Y_dev)\n",
        "# print('dev loss, dev acc:', results_dev)\n",
        "\n",
        "# print('\\n# Evaluate on test data')\n",
        "# results_test = Senet_model.evaluate(X_res_test,Y_test)\n",
        "# print('test loss, test acc:', results_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1bCigzO42FBP",
        "colab_type": "code",
        "outputId": "752fb404-0571-4fc2-c129-365887c0e84d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "Senet_model_wcw = load_model('/content/drive/My Drive/cs230 project/models/tl/SeNet50-WCW-BEST-68.9.h5')\n",
        "print('\\n# Evaluate on dev data')\n",
        "results_dev = Senet_model_wcw.evaluate(X_res_dev,Y_dev)\n",
        "print('dev loss, dev acc:', results_dev)\n",
        "\n",
        "print('\\n# Evaluate on test data')\n",
        "results_test = Senet_model_wcw.evaluate(X_res_test,Y_test)\n",
        "print('test loss, test acc:', results_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "# Evaluate on dev data\n",
            "3589/3589 [==============================] - 12s 3ms/step\n",
            "dev loss, dev acc: [1.1761809421070575, 0.6831986626133211]\n",
            "\n",
            "# Evaluate on test data\n",
            "3589/3589 [==============================] - 9s 2ms/step\n",
            "test loss, test acc: [1.099774395075971, 0.6893285037781011]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WDEz8VZ3CfGO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# VGG100_model = load_model('/content/drive/My Drive/cs230 project/models/tl/VGG-BEST-69.5.h5')\n",
        "# print('\\n# Evaluate on dev data')\n",
        "# results_dev = VGG100_model.evaluate(X_res_dev,Y_dev)\n",
        "# print('dev loss, dev acc:', results_dev)\n",
        "\n",
        "# print('\\n# Evaluate on test data')\n",
        "# results_test = VGG100_model.evaluate(X_res_test,Y_test)\n",
        "# print('test loss, test acc:', results_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W5gbMaNNEke3",
        "colab_type": "code",
        "outputId": "f0c88711-46d6-4fbe-c243-cb928299e011",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 497
        }
      },
      "source": [
        "# VGG100_model_wcw = load_model(\"/content/drive/My Drive/cs230 project/models/tl/vgg100-WCW-BEST-70.h5\")\n",
        "# print('\\n# Evaluate on dev data')\n",
        "# results_dev = VGG100_model_wcw.evaluate(X_res_dev,Y_dev)\n",
        "# print('dev loss, dev acc:', results_dev)\n",
        "\n",
        "# print('\\n# Evaluate on test data')\n",
        "# results_test = VGG100_model_wcw.evaluate(X_res_test,Y_test)\n",
        "# print('test loss, test acc:', results_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "OSError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-23-8c762c5cbac4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mVGG100_model_wcw\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/content/drive/My Drive/cs230 project/models/tl/vgg100-WCW-BEST-70.h5\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\n# Evaluate on dev data'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mresults_dev\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVGG100_model_wcw\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_res_dev\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mY_dev\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'dev loss, dev acc:'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresults_dev\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py\u001b[0m in \u001b[0;36mload_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    456\u001b[0m                 \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremove\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtmp_filepath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    457\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 458\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mload_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    459\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    460\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mload_wrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py\u001b[0m in \u001b[0;36mload_model\u001b[0;34m(filepath, custom_objects, compile)\u001b[0m\n\u001b[1;32m    547\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    548\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mH5Dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_supported_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 549\u001b[0;31m         \u001b[0;32mwith\u001b[0m \u001b[0mH5Dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mh5dict\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    550\u001b[0m             \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_deserialize_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mh5dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcustom_objects\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    551\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'write'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/utils/io_utils.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, path, mode)\u001b[0m\n\u001b[1;32m    189\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstring_types\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_is_path_instance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 191\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5py\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    192\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/h5py/_hl/files.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, mode, driver, libver, userblock_size, swmr, **kwds)\u001b[0m\n\u001b[1;32m    310\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mphil\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    311\u001b[0m                 \u001b[0mfapl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_fapl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdriver\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlibver\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 312\u001b[0;31m                 \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_fid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muserblock_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswmr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswmr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    313\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    314\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mswmr_support\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/h5py/_hl/files.py\u001b[0m in \u001b[0;36mmake_fid\u001b[0;34m(name, mode, userblock_size, fapl, fcpl, swmr)\u001b[0m\n\u001b[1;32m    140\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mswmr\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mswmr_support\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m             \u001b[0mflags\u001b[0m \u001b[0;34m|=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mACC_SWMR_READ\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 142\u001b[0;31m         \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfapl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    143\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'r+'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m         \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh5f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mACC_RDWR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfapl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfapl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mh5py/h5f.pyx\u001b[0m in \u001b[0;36mh5py.h5f.open\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mOSError\u001b[0m: Unable to open file (unable to open file: name = '/content/drive/My Drive/cs230 project/models/tl/vgg100-WCW-BEST-70.h5', errno = 2, error message = 'No such file or directory', flags = 0, o_flags = 0)"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kGWp8hPnASEr",
        "colab_type": "code",
        "outputId": "56ff1fe0-f1cc-400b-8b16-4f325f877eb1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "Resnet_Aux_model = load_model(\"/content/drive/My Drive/cs230 project/models/final/RESNET50-AUX-BEST-72.7.h5\")\n",
        "print('\\n# Evaluate on dev data')\n",
        "results_dev = Resnet_Aux_model.evaluate(X_res_dev,Y_dev)\n",
        "print('dev loss, dev acc:', results_dev)\n",
        "\n",
        "print('\\n# Evaluate on test data')\n",
        "results_test = Resnet_Aux_model.evaluate(X_res_test,Y_test)\n",
        "print('test loss, test acc:', results_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "# Evaluate on dev data\n",
            "3589/3589 [==============================] - 13s 4ms/step\n",
            "dev loss, dev acc: [0.8609743998488466, 0.7074393981942628]\n",
            "\n",
            "# Evaluate on test data\n",
            "3589/3589 [==============================] - 8s 2ms/step\n",
            "test loss, test acc: [0.8026626259711301, 0.7194204513792143]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7hGURVemBtGU",
        "colab_type": "code",
        "outputId": "142745e2-f2ef-445c-cbf5-6b35c178f549",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "Resnet_Aux_model_wcw = load_model(\"/content/drive/My Drive/cs230 project/models/final/RESNET50-WCW-AUX-BEST-72.4.h5\")\n",
        "print('\\n# Evaluate on dev data')\n",
        "results_dev = Resnet_Aux_model_wcw.evaluate(X_res_dev,Y_dev)\n",
        "print('dev loss, dev acc:', results_dev)\n",
        "\n",
        "print('\\n# Evaluate on test data')\n",
        "results_test = Resnet_Aux_model_wcw.evaluate(X_res_test,Y_test)\n",
        "print('test loss, test acc:', results_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "# Evaluate on dev data\n",
            "3589/3589 [==============================] - 12s 3ms/step\n",
            "dev loss, dev acc: [0.8581980269212316, 0.7035385901448321]\n",
            "\n",
            "# Evaluate on test data\n",
            "3589/3589 [==============================] - 8s 2ms/step\n",
            "test loss, test acc: [0.7959005565954205, 0.7227640011477318]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CQVpy32x_yoV",
        "colab_type": "code",
        "outputId": "173b2c4f-4ed2-4e71-f6c9-8e9676677982",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "Senet_Aux_model = load_model('/content/drive/My Drive/cs230 project/models/final/SENET50-AUX-BEST-72.5.h5')\n",
        "print('\\n# Evaluate on dev data')\n",
        "results_dev = Senet_Aux_model.evaluate(X_res_dev,Y_dev)\n",
        "print('dev loss, dev acc:', results_dev)\n",
        "\n",
        "print('\\n# Evaluate on test data')\n",
        "results_test = Senet_Aux_model.evaluate(X_res_test,Y_test)\n",
        "print('test loss, test acc:', results_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "# Evaluate on dev data\n",
            "3589/3589 [==============================] - 12s 3ms/step\n",
            "dev loss, dev acc: [2.0087849657560133, 0.7107829479295651]\n",
            "\n",
            "# Evaluate on test data\n",
            "3589/3589 [==============================] - 9s 2ms/step\n",
            "test loss, test acc: [1.845603608076587, 0.721928113680691]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G9WhAotjUoa7",
        "colab_type": "code",
        "outputId": "c829a7f9-bf9f-420c-b046-4bc28a0d6f4a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "Senet_Aux_model_wcw = load_model('/content/drive/My Drive/cs230 project/models/final/SENET50-WCW-AUX-BEST-71.6.h5')\n",
        "print('\\n# Evaluate on dev data')\n",
        "results_dev = Senet_Aux_model_wcw.evaluate(X_res_dev,Y_dev)\n",
        "print('dev loss, dev acc:', results_dev)\n",
        "\n",
        "print('\\n# Evaluate on test data')\n",
        "results_test = Senet_Aux_model_wcw.evaluate(X_res_test,Y_test)\n",
        "print('test loss, test acc:', results_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "# Evaluate on dev data\n",
            "3589/3589 [==============================] - 15s 4ms/step\n",
            "dev loss, dev acc: [1.8545933933469305, 0.7107829479129576]\n",
            "\n",
            "# Evaluate on test data\n",
            "3589/3589 [==============================] - 9s 3ms/step\n",
            "test loss, test acc: [1.7963547822857142, 0.7099470604791319]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "voByce__QNLw",
        "colab_type": "code",
        "outputId": "ae28cfd3-8c66-4176-a67b-1e74945e58d4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "!ls \"/content/drive/My Drive/cs230 project/models/final/\""
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "RESNET50-EPOCHS_50test_acc_0.727.h5\n",
            "RESNET50-WCW-EPOCHS_50test_acc_0.724.h5\n",
            "SENET50-EPOCHS_100test_acc_0.725.h5\n",
            "VGG16-AUX-BEST-70.2.h5\n",
            "VGG16-Class-Weights-EPOCHS_50test_acc_0.696.h5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lfsWIXEbP1Mv",
        "colab_type": "code",
        "outputId": "42327786-52ab-48c6-9a82-85eaabec4962",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "VGG_Aux_model = load_model(\"/content/drive/My Drive/cs230 project/models/final/VGG16-AUX-BEST-70.2.h5\")\n",
        "print('\\n# Evaluate on dev data')\n",
        "results_dev = VGG_Aux_model.evaluate(X_res_dev,Y_dev)\n",
        "print('dev loss, dev acc:', results_dev)\n",
        "\n",
        "print('\\n# Evaluate on test data')\n",
        "results_test = VGG_Aux_model.evaluate(X_res_test,Y_test)\n",
        "print('test loss, test acc:', results_test)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "# Evaluate on dev data\n",
            "3589/3589 [==============================] - 12s 3ms/step\n",
            "dev loss, dev acc: [0.9085961669957482, 0.6884926163442755]\n",
            "\n",
            "# Evaluate on test data\n",
            "3589/3589 [==============================] - 9s 2ms/step\n",
            "test loss, test acc: [0.8508083837780989, 0.6929506826414042]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bkG2Ff2ep-R3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "models_SOA = [model, model2]\n",
        "models_TL = [Resnet_model, Resnet_Aux_model_wcw, Senet_Aux_model, Senet_Aux_model_wcw, VGG_Aux_model]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EJqnQzpmpHBN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# make an ensemble prediction for multi-class classification\n",
        "def ensemble_predictions(models_SOA, testX, models_TL, testresX):\n",
        "  # make predictions\n",
        "  yhats = np.zeros((len(models_SOA)+len(models_TL),testX.shape[0],7))\n",
        "\n",
        "  for model_ind in range(len(models_SOA)):\n",
        "    yhat = models_SOA[model_ind].predict(testX)\n",
        "    yhats[model_ind,:,:] = yhat\n",
        "\n",
        "  for model_ind in range(len(models_TL)):\n",
        "    yhat = models_TL[model_ind].predict(testresX)\n",
        "    yhats[len(models_SOA)+model_ind,:,:] = yhat\n",
        "\n",
        "  summed = np.sum(yhats, axis=0)\n",
        "  result = np.argmax(summed, axis=1)\n",
        "  return result\n",
        " \n",
        "# evaluate a specific number of members in an ensemble\n",
        "def evaluate_n_members(models_SOA, testX, models_TL, testresX, testy):\n",
        "\t# select a subset of members\n",
        "\t#subset = members[:n_members]\n",
        "\t#print(len(subset))\n",
        "\t# make prediction\n",
        "\tyhat = ensemble_predictions(models_SOA, testX, models_TL, testresX)\n",
        "\t# calculate accuracy\n",
        "\treturn accuracy_score(testy, yhat)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BeDZdqSOiAXA",
        "colab_type": "code",
        "outputId": "aef05bb7-f031-4ac2-cd62-8fa875656769",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "ens_acc = evaluate_n_members(models_SOA, X_test, models_TL, X_res_test, np.argmax(Y_test, axis=1))\n",
        "print(ens_acc)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.7575926441905824\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7yKjgEv5z3fy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ens_acc = evaluate_n_members(models_SOA, X_dev, models_TL, X_res_dev, np.argmax(Y_dev, axis=1))\n",
        "# print(ens_acc)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L0wbYzu6qYGt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}